package com.dnb.webmash.playmaker.server;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.jdo.PersistenceManager;
import javax.jdo.Query;

import net.sf.jsr107cache.Cache;
import net.sf.jsr107cache.CacheException;
import net.sf.jsr107cache.CacheFactory;
import net.sf.jsr107cache.CacheManager;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.dnb.webmash.playmaker.client.WebPlaylistService;
import com.dnb.webmash.playmaker.shared.LoginInfo;
import com.dnb.webmash.playmaker.shared.MediaType;
import com.dnb.webmash.playmaker.shared.MyURL;
import com.dnb.webmash.playmaker.shared.PlayerConfig;
import com.dnb.webmash.playmaker.shared.PlayerType;
import com.dnb.webmash.playmaker.shared.PlaylistItem;
import com.dnb.webmash.playmaker.shared.Utility;
import com.dnb.webmash.playmaker.shared.playerconfigs.FlashSettings;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.memcache.MemcacheService;
import com.google.appengine.api.memcache.MemcacheServiceException;
import com.google.appengine.api.memcache.stdimpl.GCacheFactory;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import com.google.apphosting.api.DeadlineExceededException;
import com.google.gwt.core.client.GWT;
import com.google.gwt.user.server.rpc.RemoteServiceServlet;

@SuppressWarnings("serial")
public class WebPlaylistServiceImpl extends RemoteServiceServlet implements 
WebPlaylistService {
	private String requestedURL;
	private static final String MEDIA_CONST = "Media";
	private static final String SITEMAP_CONST = "Sitemap";
	private static final String LINKS_CONST = "Links";
	private static final int CONST_MAX_NextLevelPerRequest = 20;

	private static final Logger LOG = Logger.getLogger(WebPlaylistServiceImpl.class.getName()); //TODO USE IF NECESSARY FOR APP ENGINE SERVERSIDE LOGGING (viewable online)

	@Override
	public Text requestDoc(String reqURL){
		Text docText = null;
		try {//TRY TO LOAD IT FROM CACHE FIRST
			docText = (Text) getCachedObjectOwnedBy(reqURL);
		}catch (ClassCastException e){
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
		}
		
		if (docText == null){ //IF NOTHING LOADED...
			Document doc = requestDocument(reqURL);//REQUEST DOCUMENT USING HTTP GET
			if (doc!=null){
				docText = new Text(doc.outerHtml());
				cacheObject(docText, reqURL);	//AND STORE IT IN CASE THE SAME URL IS REQUESTED
			}
		}
		//BEWARE... docText COULD STILL BE NULL WHEN RETURNED
		return docText;
	}
	
	private Document requestDocument(String reqURL){
		Document doc = null;
		try {
			doc = Jsoup.connect(reqURL).get();	//GET THE ENTIRE HTML OF THE PAGE AT THIS URL
		} catch (IOException e) {
			LOG.log(Level.SEVERE, e.getLocalizedMessage());		}
		return doc;
	}
	@Override
	public PlaylistItem requestPlaylist(String input, int levels, Boolean mediaOnly, String requestID, Text docText, ArrayList<MediaType> allowedMedia) throws IllegalArgumentException {
		String myReqID;
		// Verify that the input is valid. 
		if (!ServerFieldVerifier.isValidURL(input)) {
			throw new IllegalArgumentException("Invalid URL!");
		}
		else requestedURL = input.trim();

		//Add Root playlistitem node for the MyURL
		PlaylistItem root = new PlaylistItem(new MyURL(this.requestedURL));//HOLDS THE PLAYLIST			

		try {							       
			LinkedHashSet<PlaylistItem> rootChildren = new LinkedHashSet<PlaylistItem>();

			PlaylistItem intLinks = new PlaylistItem(SITEMAP_CONST);	
			Set<PlaylistItem> internalLinks = new LinkedHashSet<PlaylistItem>(); //no duplicates makes for faster recursion
			PlaylistItem extLinks = new PlaylistItem(LINKS_CONST);				
			LinkedHashSet<PlaylistItem> externalLinks = new LinkedHashSet<PlaylistItem>(); //dont mind duplicates here
			PlaylistItem media = new PlaylistItem(MEDIA_CONST);			
			Set<PlaylistItem> mediaItems = new LinkedHashSet<PlaylistItem>(); //dont duplicate media, and sort according to added order

			Document doc;
			if (docText == null){// IF TEXT SUBMITTED IS NULL... LOAD THE CACHED DOC		
				Thread.currentThread();//USE THIS TO PREVENT HARDDEADLINE EXCEEDED ERRORS?
				Thread.sleep(0);
				doc = requestDocument(requestedURL);
			} else {
				doc = Jsoup.parse(docText.getValue());
			}
			if (doc == null) return null; //SOMETHING WENT WRONG

			//Get the links
			Elements links = doc.select("a[href]");
			for (Element link : links) {
				String linkHref = ServerFieldVerifier.cleanURL(link.attr("abs:href"));//clean the link to remove any arguments in the URL
				String linkText = link.text().trim();

				PlaylistItem f;
				if (!linkText.isEmpty())
					f = new PlaylistItem(new MyURL(linkHref), linkText);
				else f = new PlaylistItem(new MyURL(linkHref));

				String clean = ServerFieldVerifier.getBaseDomain(linkHref);
				if (ServerFieldVerifier.getBaseDomain(this.requestedURL).equals(clean))
					internalLinks.add(f);
				else
					externalLinks.add(f);
			}

			//GET THE MEDIA
			if ((allowedMedia == null)||(!allowedMedia.isEmpty())){
				allowedMedia = MediaLibrary.getAllTypes();
			}
			Elements objects;
			for (MediaType mT : allowedMedia){
				if (isGenericMedia(mT)){
					objects = doc.select(mT.getTestPat());
					for (Element object : objects) {
						for (String s : mT.getMediaIDPats()) {
							if (!object.select(s).isEmpty()){
								PlaylistItem gmpI = createGenericMediaPlsItem(object, mT);
								if (heuristicFilter(gmpI.getFullURL()))
									//TODO CONTINUE HERE : ADD TAGS TO THE PLAYABLE ITEMS
									mediaItems.add(gmpI);
							}
						}
					}	
				}
				else {
					if (doc.outerHtml().contains(mT.getTestPat())){
						//TODO ADD TAGS TO THE PLAYABLE ITEMS HERE
						mediaItems.addAll(PlaylistFromMediaTypes.addMediaItems(doc.outerHtml(), mT, requestedURL));
					}	
				}
			}
			
			//update the playlistitems from the lists
			intLinks.addChildren(new LinkedHashSet<PlaylistItem> (internalLinks));
			extLinks.addChildren(externalLinks);			
			media.addChildren(new LinkedHashSet<PlaylistItem> (mediaItems));								
			if (!mediaOnly){
				rootChildren.add(intLinks);
				rootChildren.add(extLinks);
			}
			if (media.hasChildren())
				rootChildren.add(media);
			if (!rootChildren.isEmpty())
				root.addChildren(rootChildren);

			//Ensure this request can be resumed			
			ArrayList <PlaylistItem> values = new ArrayList<PlaylistItem>(internalLinks);
			ArrayList <PlaylistItem> approved = new ArrayList<PlaylistItem>();
			if (mediaOnly) values.addAll(externalLinks); 
			//go into all links when just looking for media recursively... otherwise its a sitemap request and we don't care about expanding external links
			if (!values.isEmpty()){
				if (requestID == null) //NEW REQUEST, requestID can be null on first iteration of recursion
					myReqID = Utility.generateRequestID();						
				else myReqID = requestID;
				this.updateVisitedURLsCache(myReqID, root.getFullURL());//if this is the first time this node is cached, it is first request (and now its cached)
				for (PlaylistItem child : values){
					if (child!=null){
						String childsURL = child.getFullURL();					
						if ((childsURL!=null)&&(child.getURL().compareTo(root.getURL())>0)){
							//make sure recursively entered links are subfolders of the root (longer URLs) 
							//HEURISTICS HERE	 				
							if (heuristicPlsItFilter(child)){														
								if (this.updateVisitedURLsCache(myReqID, child.getFullURL()))//TODO FIX THIS NOT ALLOWING ANYTHING THROUGH
									//TODO Store the visited URLS here... only go into recursion if wasnt already there
									approved.add(child);
							}
						}
					}
				}
				root.addToNextLevel(approved); //set the nextlevel so this recursion request can be resumed
				//**********************MAIN HALF OF ENABLERECURSIVECRAWLING FINISHED>>> INTERNAL RECURSION NEXT
				if (levels>0){
					for (PlaylistItem child : approved){
						if (child!=null){
							String childsURL = child.getFullURL();	
							int nextLevel = levels-1;
							Thread.currentThread();
							Thread.sleep(0);//makes sure the request is timed properly
							PlaylistItem it = requestPlaylist(childsURL,nextLevel,mediaOnly, myReqID, null, allowedMedia);									
							if (it!=null){
								it.setName(child.getName());
								if (!mediaOnly){
									if (it.hasChildren()){
										intLinks.removeChild(child);//disown the child that hasnt grandchildren
										intLinks.addChild(it);//adopt the child with children
									}
								}
								else {
									if (it.hasChildren()){
										LinkedHashSet<PlaylistItem> list = it.getChildren();
										for (PlaylistItem p : list){
											if (p.getName().equals(MEDIA_CONST)){
												media.addChildren(p.getChildren());															
											}
										}
									}
								}
							}else break; //TIMES UP!
						}
					}
					//update the playlistitems from the lists (only if we went into internal recursion)
					intLinks.addChildren(new LinkedHashSet<PlaylistItem> (internalLinks));
					extLinks.addChildren(externalLinks);
					media.addChildren(new LinkedHashSet<PlaylistItem> (mediaItems));								
					if (!mediaOnly){
						rootChildren.add(intLinks);
						rootChildren.add(extLinks);
					}
					if (media.hasChildren())
						rootChildren.add(media);
					if (!rootChildren.isEmpty())
						root.addChildren(rootChildren);
				}
			}
		} catch (IllegalArgumentException e){
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;
		} catch (DeadlineExceededException e){
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;//avoid hard deadlines at all costs
		} catch (InterruptedException e) {
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;
		}
		return root;		
	}

	private boolean isGenericMedia(MediaType c) {
		switch (c.getPlayerType()) {
		case EMBED:
			return true;
		case SCRIPT:
			return true;
		default:
			return false;
		//case IFRAME, FILE, FLASH: return false
		}
	}

	@Override
	public PlaylistItem resumeRecursion(PlaylistItem empty, Boolean mediaOnly, String requestID, Text docText, ArrayList<MediaType> allowedMedia){
		PlaylistItem root;
		try {
			if (empty.getURL()!=null)
				root = new PlaylistItem(empty.getURL());//HOLDS THE PLAYLIST			
			else return null;

			//Copy or create new main nodes in root
			LinkedHashSet<PlaylistItem> emptyChildren = empty.getChildren();
			PlaylistItem intLinks, extLinks, media;
			media = new PlaylistItem(MEDIA_CONST);
			intLinks = new PlaylistItem(SITEMAP_CONST);
			extLinks = new PlaylistItem(LINKS_CONST);
			
			for (PlaylistItem r : emptyChildren){
				if (r.getName().equals(MEDIA_CONST))
					media.addChildren(r.getChildren());
				if (r.getName().equals(SITEMAP_CONST))
					intLinks.addChildren(r.getChildren());
				if (r.getName().equals(LINKS_CONST))
					extLinks.addChildren(r.getChildren());
			}

			//try to resume at next level
			ArrayList<PlaylistItem> nextLevel = empty.getNextLevel();
			if (nextLevel!=null){
				//trim the next level to ensure this request is not too time consuming... push the excess downward
				if (nextLevel.size()>CONST_MAX_NextLevelPerRequest){
					ArrayList<PlaylistItem> excess = new ArrayList<PlaylistItem>(nextLevel.subList(CONST_MAX_NextLevelPerRequest+1, nextLevel.size()));
					nextLevel = new ArrayList<PlaylistItem>(nextLevel.subList(0, CONST_MAX_NextLevelPerRequest));
					root.addToNextLevel(excess);
				}
				PlaylistItem q;
				for (PlaylistItem p : nextLevel){
					if (ServerFieldVerifier.isValidURL(p.getFullURL())){ 
						Thread.currentThread();
						Thread.sleep(0);//makes sure the request is timed properly
						q = requestPlaylist(p.getFullURL(),0,mediaOnly,requestID, docText, allowedMedia);
						if (q==null) break; //TIMES UP
						if (q.getNextLevel()!=null)
							root.addToNextLevel(q.getNextLevel());//make sure to keep passing the buck for each visited link
						if (q.hasChildren()){
							LinkedHashSet<PlaylistItem> list = q.getChildren();
							for (PlaylistItem r : list){ //add the main nodes
								if (r.getName().equals(MEDIA_CONST))
									media.addChildren(r.getChildren());
								if (r.getName().equals(SITEMAP_CONST))
									intLinks.addChildren(r.getChildren());
								if (r.getName().equals(LINKS_CONST))
									extLinks.addChildren(r.getChildren());
							}
						}
					}
				}
				LinkedHashSet<PlaylistItem> rootChildren = new LinkedHashSet<PlaylistItem>();

				if (!mediaOnly){
					rootChildren.add(intLinks);
					rootChildren.add(extLinks);
				}
				if (media.hasChildren()){
					rootChildren.add(media);
				}
				if (!rootChildren.isEmpty())
					root.addChildren(rootChildren);
			}
		} catch (InterruptedException e) {
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;
		}
		return root;
	}

	@Override
	public Boolean requestSaveUserPls(LinkedHashSet<PlaylistItem> pls, LoginInfo login){//TODO CLEAR PREVIOUS SAVED SESSION AND RESAVE
		PersistenceManager pm = PMF.get().getPersistenceManager();
		try {
			// todo get all the lists from each level and store them
			LinkedHashSet<PlaylistItem> store = new LinkedHashSet<PlaylistItem>();
			if (login.getUniqueID() != null) {
				// DELETE PREVIOUS SESSION FIRST        		 
				Query q = pm.newQuery(PlaylistItem.class, "owner == u");
				q.declareParameters("com.google.appengine.api.users.User u");
				q.setOrdering("createDate ascending");	    		
				@SuppressWarnings("unchecked")
				List<PlaylistItem> playlists = (List<PlaylistItem>) q.execute(login.getUniqueID());
				pm.deletePersistentAll(playlists);

				// NOW SAVE NEW ONE
				//flatten first
				for (PlaylistItem p : pls){        			
					store.add(p);
					if (p.hasChildren())
						store.addAll(getChildrenFlat(pls));        			
				}
				//now take ownership
				for (PlaylistItem p : store)
					p.setOwner(login.getUniqueID());
				//and send to datastore
				pm.makePersistentAll(store);
			} else {
				GWT.log("NULL USER - CANT SAVE PLAYLIST");
				return new Boolean(false);
			}
		} finally {
			pm.close();
		}
		//TODO ADD VERIFICATION THAT SAVING WORKED
		return new Boolean(true);        
	}

	@Override
	public PlaylistItem requestLoadCurUserPls() {
		UserService userService = UserServiceFactory.getUserService();
		String userID = userService.getCurrentUser().getUserId();
		if (userID!=null){
			return requestLoadPlsHelper(userID, "LoadedPreviousSession");
		}
		else
			return null;		//TODO deal with an invalid load request when user is not logged in
	}
	@Override
	public PlaylistItem requestLoadGlobalPls() {
		return requestLoadPlsHelper("global", "WhatEveryoneIsWatching");//TODO needs limits to # of results, cleanup, top10?
	}
	public ArrayList<PlaylistItem> requestLoadPls(ArrayList<String> request){
		ArrayList<PlaylistItem> result = new ArrayList<PlaylistItem>();
		for (String r : request){
			result.add(this.requestLoadPlsHelper(r, r));
		}
		return result;
	}
	@Override
	public ArrayList<MediaType> requestAllPlayTypes() {
		return MediaLibrary.getAllTypes();
	}
	@Override
	public PlayerConfig requestDefaultConfigs(PlayerType type) {
		// TODO IMPLEMENT PROPERLY ... USE MEDIALIBRARY to list all pref types (have to change it a little due to MediaType hardcoded in there)
		// then cycle all the pref types and return matching one

		//TESTING
		if (type == PlayerType.FLASH){
			PlayerConfig result = new FlashSettings();
			return result;
		}
		else
			return null;
	}
	@Override
	public PlayerConfig requestLoadUserConfigs(PlayerType type) {//IF NOTHING TO LOAD RETURN DEFAULTS
		// TODO IMPLEMENT!

		//TESTING
		return this.requestDefaultConfigs(type);
	}

	@Override
	public Boolean storeIfNotAlready(ArrayList<PlaylistItem> plsToStore, String name) {
		boolean storedSomething = false;
		if (name!=null) //store the requested URL
			storedSomething = storeIfNotAlready(new PlaylistItem(new MyURL(name)),"global");
		for (PlaylistItem p : plsToStore){			
			storedSomething = storeIfNotAlready(p,p.getMediatype());//STORE TO SELF OWNERSHIP... mediatype name owns it
		}
		return storedSomething;
	}

	@Override
	public boolean requestCreateNewMedia(MediaType nT) {//TODO ERROR CATCHING... PREVENT GARBAGE?
		if (nT!=null){
			String owner = new String("Media");
			PersistenceManager pm = PMF.get().getPersistenceManager();
			boolean result = false;
			try {
				nT.setOwner(owner);//TODO add adult filters here? store to different owner.		
				pm.makePersistent(nT);
				result = true;
			} finally {
				pm.close();
			}
			return result;
		}
		else return false;
	}

	/****************************************************************************************************************
	 * END OF RPC Methods
	 * Start of Helpers
	 ****************************************************************************************************************/
	@SuppressWarnings("unchecked")
	public PlaylistItem requestLoadPlsHelper(String user, String nodeName) {
		if (!user.equals(null)){					
			LinkedHashSet<PlaylistItem> result = new LinkedHashSet<PlaylistItem>();
			PersistenceManager pm = PMF.get().getPersistenceManager();
			try {	    		
				Query q = pm.newQuery(PlaylistItem.class, "owner == u");
				q.declareParameters("com.google.appengine.api.users.User u");
				q.setOrdering("createDate ascending");	    		
				List<PlaylistItem> playlists = (List<PlaylistItem>) q.execute(user);
				for (PlaylistItem pls : playlists) {
					if ((pls!=null)&&(!pls.isSkeleton()))//DON'T LOAD GARBAGE
						result.add(pls);
				}
			} finally {
				pm.close();
			}
			PlaylistItem loadedNode = new PlaylistItem(new MyURL(nodeName), nodeName);
			//give it a URL so it is considered Unique TODO FIX THIS HACK
			loadedNode.setOwner(user);
			loadedNode.addChildren(result);
			return loadedNode;
		}
		else return null;
	}
	//returns true if it performed a makepersistent operation (if it made a change to jdo)
	private boolean storeIfNotAlready(PlaylistItem plsToStore, String owner){
		//Store every requested URL if not already there
		PersistenceManager pm = PMF.get().getPersistenceManager();
		boolean result = false;
		try {
			Query q = pm.newQuery(PlaylistItem.class);
			//q.setFilter("myURL.myValue == m && owner == u");
			q.setFilter("owner == u && (myURL.myValue.contains(m) || m.contains(myURL.myValue))");
			q.declareParameters("String m, com.google.appengine.api.users.User u");
			q.setOrdering("createDate ascending");	
			@SuppressWarnings("unchecked")
			List<PlaylistItem> playlists = (List<PlaylistItem>) q.execute(plsToStore.getFullURL(), owner);
			if (playlists.isEmpty()){
				result = true;    			
				plsToStore.setOwner(owner);//TODO add adult filters here? store to different owner.		
				pm.makePersistent(plsToStore);
			}
			else result = false;
		} finally {
			pm.close();
		}
		return result;
	}

	/**
	 * Used to maintain a list of visited URLs so that they are only visited once per request.
	 * @param requestID The given session ID to keep track of this request
	 * @param myURL The URL to be added to the list of visited URLs. 
	 * @return true if the cache was updated by adding a URL to the list of visited URLs for this request, false if the URL was not added
	 */
	@SuppressWarnings("unchecked")
	private boolean updateVisitedURLsCache(String requestID, String myURL){
		Object cached = this.getCachedObjectOwnedBy(requestID);
		ArrayList<String> visitedURLS;
		if ((cached!=null)&&(cached instanceof ArrayList)){
			try {
				visitedURLS = (ArrayList<String>) cached;
				if (visitedURLS.contains(myURL)) return false;
				else {
					visitedURLS.add(myURL);
					return this.cacheObject(visitedURLS, requestID);
				}
			} catch (ClassCastException e) {
				LOG.log(Level.SEVERE, e.getLocalizedMessage());
				return false;
			}
		} else {
			visitedURLS = new ArrayList<String>();
			visitedURLS.add(myURL);
			return this.cacheObject(visitedURLS, requestID);			
		}
	}
	
	/**
	 * @param obj The value to be stored
	 * @param owner The key to store it under
	 * @return true if stored successfully, false otherwise
	 */
	@SuppressWarnings({ "rawtypes", "unchecked" })
	private boolean cacheObject(Object obj, String owner) {
		try {
			Map props = new HashMap();
			props.put(GCacheFactory.EXPIRATION_DELTA, 3600); //Hardcoded one hour cache lifetime TODO change this depending on request
			props.put(MemcacheService.SetPolicy.SET_ALWAYS, true);       

			CacheFactory cacheFactory = CacheManager.getInstance().getCacheFactory();
			Cache cache = cacheFactory.createCache(props);

			cache.put(owner, obj);//TODO IS THERE ANY WAY TO VERIFY IF THIS WORKED?
			return true;
		} catch (CacheException e) {
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return false;
		} catch (MemcacheServiceException e){
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return false;
		}
	}

	@SuppressWarnings({ "rawtypes" })
	private Object getCachedObjectOwnedBy(String owner) {
		try {
			CacheFactory cacheFactory = CacheManager.getInstance().getCacheFactory();
			Cache cache = cacheFactory.createCache(new HashMap());

			if (cache.containsKey(owner)){
				return cache.get(owner);
			}
			else return null;
		} catch (CacheException e) {
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;
		} catch (MemcacheServiceException e){
			LOG.log(Level.SEVERE, e.getLocalizedMessage());
			return null;
		}
	}



	private PlaylistItem createGenericMediaPlsItem(Element obj, MediaType type) {
		Element object = resolveAbsoluteLinks(obj);
		String url = this.findURL(object);
		String html = object.outerHtml();
		PlaylistItem f = new PlaylistItem(new MyURL(url));
		f.setElement(html);					  
		f.setMediatype(type.getName());
		f.setPlayerType(type.getPlayerType());
		f.setPlayable(true);
		return f;
	}
	
	/**
	 * Resolves all relative attribute values into absolute URIs to enable generic media extraction
	 * @param object The element to resolve
	 * @return The resolved element
	 */
	private Element resolveAbsoluteLinks(Element object){
		Element result = object.clone();
		String absAttr;
		for (Element el : result.getAllElements()){
			for (Attribute at : el.attributes()){
				absAttr = el.attr("abs:"+at.getKey());
				if (!absAttr.isEmpty()) at.setValue(absAttr);
			}
		}
		return result;
	}
	
	private String findURL(Element object) {
		Elements els = object.getAllElements();
		for (Element el : els){
			Attributes attr = el.attributes();
			for (Attribute att : attr)
			{				if (ServerFieldVerifier.isValidURL(att.getValue())) return att.getValue();
			}		
		}		
		return object.baseUri();//TODO find some better alternative?
	}

	//recursively flattens a playlist
	private LinkedHashSet<PlaylistItem> getChildrenFlat( LinkedHashSet<PlaylistItem> pls) {
		LinkedHashSet <PlaylistItem> result = new LinkedHashSet <PlaylistItem>();
		for (PlaylistItem p : pls){
			result.add(p);
			if (p.hasChildren())			
				result.addAll(getChildrenFlat(p.getChildren()));							
		}
		return result;
	}
	private boolean heuristicPlsItFilter(PlaylistItem child){
		//Check for eliminating factors first
		if (child.isPlayable()) return false; //dont need to go further here
		if (child.isSkeleton()) return false; //nothing here		
		if (child.getFullURL().contains("#")) return false; //no inline links
		//	TODO Adult filters here
		//URL FILTERS HERE //TODO EXTRACT THESE AND MAKE UI		
		if (child.hasChildren()) return true; //dont cut off branches, only leaves
		return heuristicFilter(child.getFullURL());
	}
	//TODO CREATE UI FOR THIS??
	private boolean heuristicFilter(String testStr) {
		if ((testStr.toLowerCase().contains("twitter")) || (testStr.toLowerCase().contains("feed")) || (testStr.toLowerCase().contains("subscribe")) || 
				(testStr.toLowerCase().contains("rss")) || (testStr.toLowerCase().contains("login")) || (testStr.toLowerCase().contains("logout")) ||
				(testStr.toLowerCase().contains("rearrange")) || (testStr.toLowerCase().contains("navbar")) || (testStr.toLowerCase().contains("plugin")) ||
				(testStr.toLowerCase().contains("gadget")) || (testStr.toLowerCase().contains("delete")) ||
				(testStr.toLowerCase().contains("email")) || (testStr.toLowerCase().contains("edit")) || (testStr.toLowerCase().contains("blog-post-reactions"))){
			return false;
		}
		if ((testStr.toLowerCase().contains("media")) || (testStr.toLowerCase().contains("movie")) || 
				(testStr.toLowerCase().contains("video")) || (testStr.toLowerCase().contains("show")) ||
				(testStr.toLowerCase().contains("view")) || (testStr.toLowerCase().contains("embed")) ||
				(testStr.toLowerCase().contains("stream")) || (testStr.toLowerCase().contains("1")) ||
				(testStr.toLowerCase().contains("2")) || (testStr.toLowerCase().contains("3")) || 
				(testStr.toLowerCase().contains("4")) || (testStr.toLowerCase().contains("5")) || 
				(testStr.toLowerCase().contains("6")) || (testStr.toLowerCase().contains("7")) || 
				(testStr.toLowerCase().contains("8")) || (testStr.toLowerCase().contains("9")) ||
				(testStr.toLowerCase().contains("0")) || (testStr.toLowerCase().contains("watch")) ||
				(testStr.toLowerCase().contains("season")) || (testStr.toLowerCase().contains("episode")) ||
				(testStr.toLowerCase().contains("music")) || (testStr.toLowerCase().contains(".avi")) ||
				(testStr.toLowerCase().contains(".mpg")) || (testStr.toLowerCase().contains(".mpeg")) ||
				(testStr.toLowerCase().contains(".mp3")) || (testStr.toLowerCase().contains(".mp4")) ||
				(testStr.toLowerCase().contains(".mov")) || (testStr.toLowerCase().contains("flash")) ||
				(testStr.toLowerCase().contains(".rm")) || (testStr.toLowerCase().contains(".wmv")) ||			
				(testStr.toLowerCase().contains("photo")) ||(testStr.toLowerCase().contains("display")))
			// TODO TEST AND ADD MORE?
			//TODO ADD ANY DATE
			return true;

		//TODO USE ALL THE MEDIATYPE TEST PATTERNS HERE TOO
		//DEFAULT TO TRUE TODO CREATE better heuristics and default to false? 
		return true;
	}

}
